package exp;

import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;

import tools.FileReaderEnhance;
import tools.ObjectTools;
import tools.twitter.TweetTidy;
import tools.twitter.URLTools;

/**
 * Extract all URLs to download
 * 
 * @author Lanjun
 * 
 */
public class EXP3 {
	static String path = "data/plaintext1/";

	@SuppressWarnings("unchecked")
	public static void main(String[] args) {
		for (File file : (new File(path)).listFiles()) {
			String[] lines = FileReaderEnhance.readToString(file, "UTF-8")
					.split("\n");
			HashMap<String, String> urlAndContents = null;
			File urlFile = new File("data/objs/urlContent/" + file.getName() + ".dat");
			if (urlFile.exists()) {
				urlAndContents = (HashMap<String, String>) ObjectTools.readFromFile(urlFile);
			} else {
				urlAndContents = new HashMap<>();
			}

			for (String line : lines) {
				line = TweetTidy.doTidyHTML(line);
				ArrayList<String> urls = URLTools.getURLs(line);
				for (String url : urls) {
					if (urlAndContents.containsKey(url)) {
						System.out.println("Existed:" + url);
						continue;
					} else {
						urlAndContents.put(url, null);
					}
				}
			}

			ObjectTools.writeToFile(urlAndContents, "data/objs/urlContent/" + file.getName() + ".dat");
		}
	}
}
